#!/usr/bin/env python
# -*- coding:utf-8 -*-

"""
@author zyx
@since 2022/1/26 20:33
@file: scxk_query.py
@requirement: pandas, requests, xlrd1.2.0(版本过高有问题)
"""
import random
from functools import wraps

import requests
import time
import pandas as pd
from openpyxl import load_workbook

url = "http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do"
head = {  # 存放需要伪装的头信息
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
page_limit = 382
retries = 5

order = [
    "epsName",
    "legalPerson",
    "certStr",
    "epsAddress",
    "epsProductAddress",
    "qfManagerName",
    "rcManagerDepartName",
    "rcManagerUser",
    "xkDate",
    "xkDateStr"
]

columns_map = {
    "epsName": "企业名称",
    "legalPerson": "法人",
    "certStr": "生产许可项目",
    "epsAddress": "企业地址",
    "epsProductAddress": "企业地址",
    "qfManagerName": "发证机关",
    "rcManagerDepartName": "发证部门",
    "rcManagerUser": "发证人员",
    "xkDate": "有效期",
    "xkDateStr": "发证日期",
}


def timmer(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        start = time.time()
        res = func(*args, **kwargs)
        end = time.time()
        print(f'cost {end - start} s')
        return res

    return wrapper


@timmer
def get_ids(page):
    body = {
        "method": "getXkzsList",
        "page": page
    }
    i = 0
    while i < retries:
        try:
            # 设置超时时间
            resp_json = requests.post(url=url, headers=head, data=body, timeout=0.5).json()
            return list(map(lambda x: x.get('ID'), resp_json['list']))
        except Exception as e:
            time.sleep(round(random.uniform(0.1, 0.3), 2))
            print(e)
            i += 1
    return None


@timmer
def get_id(pid):
    body = {
        "method": "getXkzsById",
        "id": pid
    }
    # 重试
    i = 0
    while i < retries:
        try:
            # 设置超时时间
            resp_json = requests.post(url=url, headers=head, data=body, timeout=0.5).json()
            return resp_json
        except Exception as e:
            print(e)
            i += 1
    return None


def test():
    for i in range(1, page_limit + 1):
        print(i, get_ids(i))


def export_to_excel(dic_data, fn, sn):
    """
    追加写入 list(dict)文件到excel中
    运行前需要在当前文件夹下创建一个含有 sn 工作簿的名为 fn(后缀名为.xlsx) 的excel文件
    """
    df = pd.DataFrame(list(dic_data))
    df.rename(columns=columns_map, inplace=True)
    # 指定生成的Excel表格名称
    writer = pd.ExcelWriter(fn, engine='openpyxl')
    # 替换空单元格
    df.fillna(' ', inplace=True)
    book = load_workbook(fn)
    # 读取原数据文件和表
    reader = pd.DataFrame(pd.read_excel(fn, sheet_name=sn))
    writer.book = book
    writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
    # 获取原数据的行数
    df_rows = reader.shape[0]
    # 输出
    if df_rows == 0:
        # 第一次输入
        df.to_excel(writer, encoding='utf-8', index=False)
    else:
        # 非第一次输入
        df.to_excel(writer, encoding='utf-8', sheet_name=sn, startrow=df_rows + 1, index=False, header=False)
    # 保存表格
    writer.save()


def main():
    # 爬取list列表
    for p in range(1, page_limit + 1):
        p_lst = get_ids(p)
        # 防止取不到数据
        if p_lst is not None:
            export_list = []
            for pid in p_lst:
                # 爬取单个id对应的商品
                i_detail = get_id(pid)
                if i_detail is not None:
                    info = {}
                    for key in order:
                        info[key] = i_detail.get(key)
                    time.sleep(round(random.uniform(0.1, 0.3), 2))
                    # 整合信息输出
                    print(info)
                    export_list.append(info)
                else:
                    print(f'{pid} is None...')
            export_to_excel(export_list, 'scxk.xlsx', 'Sheet1')
            # 随机睡眠防止爬取过快
            time.sleep(round(random.uniform(0.1, 0.3), 2))


if __name__ == '__main__':
    main()
